{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## System Basics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:09:11.083145Z",
     "start_time": "2020-01-28T19:09:09.887309Z"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "print(sys.executable)\n",
    "!python --version"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:09:15.284774Z",
     "start_time": "2020-01-28T19:09:11.085363Z"
    }
   },
   "outputs": [],
   "source": [
    "!conda info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Memory Usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:09:15.315106Z",
     "start_time": "2020-01-28T19:09:15.287240Z"
    }
   },
   "outputs": [],
   "source": [
    "import psutil, os\n",
    "process = psutil.Process(os.getpid())\n",
    "process.memory_info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Memory Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:09:22.813283Z",
     "start_time": "2020-01-28T19:09:15.316851Z"
    }
   },
   "outputs": [],
   "source": [
    "!pip install memory_profiler\n",
    "%load_ext memory_profiler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:09:22.819871Z",
     "start_time": "2020-01-28T19:09:22.815589Z"
    }
   },
   "outputs": [],
   "source": [
    "import psutil\n",
    "psutil.virtual_memory()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:09:23.135534Z",
     "start_time": "2020-01-28T19:09:22.821750Z"
    }
   },
   "outputs": [],
   "source": [
    "%memit some_str = ' ' * 100000000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Numpy Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:10:41.781953Z",
     "start_time": "2020-01-28T19:10:41.776004Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.show_config()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-20T20:20:25.108641Z",
     "start_time": "2019-05-20T20:20:25.103061Z"
    }
   },
   "source": [
    "### MKL Support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:12:20.974696Z",
     "start_time": "2020-01-28T19:12:20.970495Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.python.framework import test_util\n",
    "test_util.IsMklEnabled()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:12:24.229828Z",
     "start_time": "2020-01-28T19:12:24.226076Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "print(torch.backends.mkl.is_available())\n",
    "\n",
    "# svd is only supported on MKL?\n",
    "# torch.svd(torch.tensor(np.random.randn(10,10),  dtype=torch.float))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:12:25.430591Z",
     "start_time": "2020-01-28T19:12:25.425910Z"
    }
   },
   "outputs": [],
   "source": [
    "import mxnet.runtime\n",
    "fs=mxnet.runtime.Features()\n",
    "fs.is_enabled('MKLDNN')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Numpy Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:13:01.248490Z",
     "start_time": "2020-01-28T19:12:27.840833Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from time import time\n",
    "\n",
    "print(\"Start\")\n",
    "# Let's take the randomness out of random numbers (for reproducibility)\n",
    "np.random.seed(0)\n",
    "\n",
    "size = 4096\n",
    "A, B = np.random.random((size, size)), np.random.random((size, size))\n",
    "C, D = np.random.random((size * 128,)), np.random.random((size * 128,))\n",
    "E = np.random.random((int(size / 2), int(size / 4)))\n",
    "F = np.random.random((int(size / 2), int(size / 2)))\n",
    "F = np.dot(F, F.T)\n",
    "G = np.random.random((int(size / 2), int(size / 2)))\n",
    "\n",
    "# Matrix multiplication\n",
    "N = 20\n",
    "t = time()\n",
    "for i in range(N):\n",
    "    np.dot(A, B)\n",
    "delta = time() - t\n",
    "print('Dotted two %dx%d matrices in %0.2f s.' % (size, size, delta / N))\n",
    "del A, B\n",
    "\n",
    "# Vector multiplication\n",
    "N = 5000\n",
    "t = time()\n",
    "for i in range(N):\n",
    "    np.dot(C, D)\n",
    "delta = time() - t\n",
    "print('Dotted two vectors of length %d in %0.2f ms.' % (size * 128, 1e3 * delta / N))\n",
    "del C, D\n",
    "\n",
    "# Singular Value Decomposition (SVD)\n",
    "N = 3\n",
    "t = time()\n",
    "for i in range(N):\n",
    "    np.linalg.svd(E, full_matrices = False)\n",
    "delta = time() - t\n",
    "print(\"SVD of a %dx%d matrix in %0.2f s.\" % (size / 2, size / 4, delta / N))\n",
    "del E\n",
    "\n",
    "# Cholesky Decomposition\n",
    "N = 3\n",
    "t = time()\n",
    "for i in range(N):\n",
    "    np.linalg.cholesky(F)\n",
    "delta = time() - t\n",
    "print(\"Cholesky decomposition of a %dx%d matrix in %0.2f s.\" % (size / 2, size / 2, delta / N))\n",
    "\n",
    "# Eigendecomposition\n",
    "t = time()\n",
    "for i in range(N):\n",
    "    np.linalg.eig(G)\n",
    "delta = time() - t\n",
    "print(\"Eigendecomposition of a %dx%d matrix in %0.2f s.\" % (size / 2, size / 2, delta / N))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Numba Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:13:02.846574Z",
     "start_time": "2020-01-28T19:13:01.251088Z"
    }
   },
   "outputs": [],
   "source": [
    "from numba import jit\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "x = np.arange(100).reshape(10, 10)\n",
    "\n",
    "@jit(nopython=True)\n",
    "def go_fast(a): # Function is compiled and runs in machine code\n",
    "    trace = 0\n",
    "    for i in range(a.shape[0]):\n",
    "        trace += np.tanh(a[i, i])\n",
    "    return a + trace\n",
    "\n",
    "# DO NOT REPORT THIS... COMPILATION TIME IS INCLUDED IN THE EXECUTION TIME!\n",
    "start = time.time()\n",
    "go_fast(x)\n",
    "end = time.time()\n",
    "print(\"Elapsed (with compilation) = %s\" % (end - start))\n",
    "\n",
    "# NOW THE FUNCTION IS COMPILED, RE-TIME IT EXECUTING FROM CACHE\n",
    "start = time.time()\n",
    "go_fast(x)\n",
    "end = time.time()\n",
    "print(\"Elapsed (after compilation) = %s\" % (end - start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Jax Lib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:13:04.113078Z",
     "start_time": "2020-01-28T19:13:02.848905Z"
    }
   },
   "outputs": [],
   "source": [
    "import jax.numpy as np\n",
    "from jax import jit\n",
    "\n",
    "def slow_f(x):\n",
    "  # Element-wise ops see a large benefit from fusion\n",
    "  return x * x + x * 2.0\n",
    "\n",
    "x = np.ones((5000, 5000))\n",
    "fast_f = jit(slow_f)\n",
    "%timeit -n10 -r3 fast_f(x)\n",
    "%timeit -n10 -r3 slow_f(x) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensorflow / Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:13:49.150871Z",
     "start_time": "2020-01-28T19:13:46.431008Z"
    }
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import time\n",
    "\n",
    "import tensorflow.compat.v1 as tf\n",
    "tf.disable_v2_behavior()\n",
    "\n",
    "def get_times(maximum_time):\n",
    "\n",
    "    device_times = {\n",
    "        # \"/gpu:0\":[],\n",
    "        \"/cpu:0\":[]\n",
    "    }\n",
    "    matrix_sizes = range(500,50000,50)\n",
    "\n",
    "    for size in matrix_sizes:\n",
    "        for device_name in device_times.keys():\n",
    "\n",
    "            print(\"####### Calculating on the \" + device_name + \" #######\")\n",
    "\n",
    "            shape = (size,size)\n",
    "            data_type = tf.float16\n",
    "            with tf.device(device_name):\n",
    "                r1 = tf.random.uniform(shape=shape, minval=0, maxval=1, dtype=data_type)\n",
    "                r2 = tf.random.uniform(shape=shape, minval=0, maxval=1, dtype=data_type)\n",
    "                dot_operation = tf.matmul(r2, r1)\n",
    "\n",
    "\n",
    "            with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as session:\n",
    "                    start_time = time.time()\n",
    "                    result = session.run(dot_operation)\n",
    "                    time_taken = time.time() - start_time\n",
    "                    print(result)\n",
    "                    device_times[device_name].append(time_taken)\n",
    "\n",
    "            print(device_times)\n",
    "\n",
    "            if time_taken > maximum_time:\n",
    "                return device_times, matrix_sizes\n",
    "\n",
    "\n",
    "device_times, matrix_sizes = get_times(1.5)\n",
    "if \"/gpu:0\" in device_times:\n",
    "    gpu_times = device_times[\"/gpu:0\"]\n",
    "    plt.plot(matrix_sizes[:len(gpu_times)], gpu_times, 'o-', color='blue')\n",
    "\n",
    "cpu_times = device_times[\"/cpu:0\"]\n",
    "plt.plot(matrix_sizes[:len(cpu_times)], cpu_times, 'o-', color='green')\n",
    "plt.ylabel('Time')\n",
    "plt.xlabel('Matrix size')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:13:46.428453Z",
     "start_time": "2020-01-28T19:13:21.791210Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "mnist = tf.keras.datasets.mnist\n",
    "\n",
    "(x_train, y_train),(x_test, y_test) = mnist.load_data()\n",
    "x_train, x_test = x_train / 255.0, x_test / 255.0\n",
    "\n",
    "model = tf.keras.models.Sequential([\n",
    "  tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "  tf.keras.layers.Dense(512, activation=tf.nn.relu),\n",
    "  tf.keras.layers.Dropout(0.2),\n",
    "  tf.keras.layers.Dense(10, activation=tf.nn.softmax)\n",
    "])\n",
    "model.compile(optimizer='adam',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(x_train, y_train, epochs=5)\n",
    "model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:14:04.063928Z",
     "start_time": "2020-01-28T19:14:00.722550Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "class Model(nn.Module):\n",
    "    \"A typical CNN text classification model.\"\n",
    "    def __init__(self, vocab_size, num_classes, embedding_dim=100, \n",
    "                 kernel_sizes=[3,4,5],  num_filters=64, dropout_prob=0.2):\n",
    "        super(Model, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)  # embedding layer\n",
    "        self.convs = nn.ModuleList([\n",
    "            nn.Conv1d(embedding_dim, num_filters, k) for k in kernel_sizes])\n",
    "        self.dropout = nn.Dropout(dropout_prob)  # a dropout layer\n",
    "        self.fc1 = nn.Linear(len(kernel_sizes)*num_filters, num_classes)  # a dense layer for classification\n",
    "\n",
    "    @staticmethod\n",
    "    def conv_and_max_pool(x, conv):\n",
    "        \"\"\"Convolution and global max pooling layer\"\"\"\n",
    "        return F.relu(conv(x).permute(0, 2, 1).max(1)[0])\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        # Conv1d takes in (batch, channels, seq_len), but raw embedded is (batch, seq_len, channels)\n",
    "        embedded = self.embedding(inputs).permute(0, 2, 1)\n",
    "        x = [self.conv_and_max_pool(embedded, k) for k in self.convs]  # convolution and global max pooling\n",
    "        x = self.fc1(self.dropout(torch.cat(x, 1)))  # concatenation and dropout\n",
    "        return x\n",
    "\n",
    "vocab_size = 1000\n",
    "seq_length = 200\n",
    "batch_size = 8\n",
    "\n",
    "input = torch.tensor(np.random.randint(0, 1000, size=(batch_size, seq_length)))\n",
    "model = Model(vocab_size, 10, num_filters=64, embedding_dim=128, dropout_prob=0.1)\n",
    "\n",
    "# Should be a few milliseconds\n",
    "%timeit model(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:14:04.755205Z",
     "start_time": "2020-01-28T19:14:04.066020Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "device = torch.device('cpu')\n",
    "# device = torch.device('cuda') # Uncomment this to run on GPU\n",
    "\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# Create random input and output data\n",
    "x = torch.randn(N, D_in, device=device)\n",
    "y = torch.randn(N, D_out, device=device)\n",
    "\n",
    "# Randomly initialize weights\n",
    "w1 = torch.randn(D_in, H, device=device)\n",
    "w2 = torch.randn(H, D_out, device=device)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # Forward pass: compute predicted y\n",
    "    h = x.mm(w1)\n",
    "    h_relu = h.clamp(min=0)\n",
    "    y_pred = h_relu.mm(w2)\n",
    "    \n",
    "    # Compute and print loss; loss is a scalar, and is stored in a PyTorch Tensor\n",
    "    # of shape (); we can get its value as a Python number with loss.item().\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss.item())\n",
    "    \n",
    "    # Backprop to compute gradients of w1 and w2 with respect to loss\n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    grad_w2 = h_relu.t().mm(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.mm(w2.t())\n",
    "    grad_h = grad_h_relu.clone()\n",
    "    grad_h[h < 0] = 0\n",
    "    grad_w1 = x.t().mm(grad_h)\n",
    "    \n",
    "    # Update weights using gradient descent\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:14:15.325747Z",
     "start_time": "2020-01-28T19:14:14.042687Z"
    }
   },
   "outputs": [],
   "source": [
    "import shap\n",
    "import xgboost\n",
    "X,y = shap.datasets.boston()\n",
    "model = xgboost.train({\"learning_rate\": 0.01}, xgboost.DMatrix(X, label=y), 100)\n",
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MXNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:14:16.777981Z",
     "start_time": "2020-01-28T19:14:16.771615Z"
    }
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "a = mx.nd.ones((2, 3))\n",
    "b = a * 2 + 1\n",
    "b.asnumpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fastai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:14:42.891005Z",
     "start_time": "2020-01-28T19:14:19.051529Z"
    }
   },
   "outputs": [],
   "source": [
    "from fastai.script import *\n",
    "from fastai.vision import *\n",
    "from fastai.distributed import *\n",
    "\n",
    "path = untar_data(URLs.MNIST_SAMPLE)\n",
    "tfms = (rand_pad(2, 28), [])\n",
    "data = ImageDataBunch.from_folder(path, ds_tfms=tfms, bs=64).normalize(imagenet_stats)\n",
    "learn = cnn_learner(data, models.resnet18, metrics=accuracy)\n",
    "learn.fit_one_cycle(1, 0.02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:35:01.182926Z",
     "start_time": "2020-01-28T19:35:01.096897Z"
    }
   },
   "outputs": [],
   "source": [
    "import fastai.utils\n",
    "fastai.utils.show_install(True)\n",
    "\n",
    "# Temp workaround to fix missing PILLOW_VERSION on 7.0.0\n",
    "from PIL import Image\n",
    "Image.PILLOW_VERSION = PIL.__version__\n",
    "\n",
    "fastai.utils.check_perf()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Theano"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:14:49.312579Z",
     "start_time": "2020-01-28T19:14:43.405703Z"
    }
   },
   "outputs": [],
   "source": [
    "import theano\n",
    "import numpy\n",
    " \n",
    "x = theano.tensor.fvector('x')\n",
    "W = theano.shared(numpy.asarray([0.2, 0.7]), 'W')\n",
    "y = (x * W).sum()\n",
    " \n",
    "f = theano.function([x], y)\n",
    " \n",
    "output = f([1.0, 1.0])\n",
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fasttext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:14:49.331045Z",
     "start_time": "2020-01-28T19:14:49.315027Z"
    }
   },
   "outputs": [],
   "source": [
    "# fasttext via python api\n",
    "import fasttext\n",
    "help(fasttext)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:15:00.751284Z",
     "start_time": "2020-01-28T19:14:49.332836Z"
    }
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "spacy.prefer_gpu()\n",
    "nlp = spacy.load('en')\n",
    "doc = nlp(u'Apple is looking at buying U.K. startup for $1 billion')\n",
    "for token in doc:\n",
    "    print(token.text, token.lemma_, token.pos_, token.tag_, token.dep_,\n",
    "          token.shape_, token.is_alpha, token.is_stop)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPU Support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:15:02.166601Z",
     "start_time": "2020-01-28T19:15:00.753222Z"
    }
   },
   "outputs": [],
   "source": [
    "# Basic GPU test\n",
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:15:03.571163Z",
     "start_time": "2020-01-28T19:15:02.168732Z"
    }
   },
   "outputs": [],
   "source": [
    "# Cuda Version\n",
    "!nvcc --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:15:10.378127Z",
     "start_time": "2020-01-28T19:15:10.038663Z"
    }
   },
   "outputs": [],
   "source": [
    "import subprocess\n",
    "gpus = []\n",
    "sp = subprocess.Popen(['nvidia-smi', '-q'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n",
    "out_str = sp.communicate()\n",
    "out_list = out_str[0].decode(\"utf-8\").split('\\n')\n",
    "out_dict = {}\n",
    "count_gpu = 0\n",
    "for item in out_list:\n",
    "    try:\n",
    "        key, val = item.split(':')\n",
    "        key, val = key.strip(), val.strip()\n",
    "        if key == 'Product Name':\n",
    "            count_gpu += 1\n",
    "            gpus.append(val)\n",
    "        out_dict[key + \"_\" + str(count_gpu)] = val\n",
    "    except:\n",
    "        pass\n",
    "gpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Theano GPU Support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:15:19.535697Z",
     "start_time": "2020-01-28T19:15:13.520625Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"THEANO_FLAGS\"] = 'device=cuda,floatX=float32'\n",
    "from theano import function, config, shared, tensor\n",
    "import numpy\n",
    "import time\n",
    "\n",
    "vlen = 10 * 30 * 768  # 10 x #cores x # threads per core\n",
    "iters = 1000\n",
    "\n",
    "rng = numpy.random.RandomState(22)\n",
    "x = shared(numpy.asarray(rng.rand(vlen), config.floatX))\n",
    "f = function([], tensor.exp(x))\n",
    "print(f.maker.fgraph.toposort())\n",
    "t0 = time.time()\n",
    "for i in range(iters):\n",
    "    r = f()\n",
    "t1 = time.time()\n",
    "print(\"Looping %d times took %f seconds\" % (iters, t1 - t0))\n",
    "print(\"Result is %s\" % (r,))\n",
    "if numpy.any([isinstance(x.op, tensor.Elemwise) and\n",
    "              ('Gpu' not in type(x.op).__name__)\n",
    "              for x in f.maker.fgraph.toposort()]):\n",
    "    print('Used the cpu')\n",
    "else:\n",
    "    print('Used the gpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Tensorflow GPU Support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:15:22.441051Z",
     "start_time": "2020-01-28T19:15:22.370125Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:16:02.872119Z",
     "start_time": "2020-01-28T19:16:02.868437Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "print(tf.config.list_physical_devices('GPU'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-27T12:12:15.362038Z",
     "start_time": "2019-06-27T12:12:15.358621Z"
    }
   },
   "source": [
    "### Test PyTorch GPU Support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:16:06.834758Z",
     "start_time": "2020-01-28T19:16:06.829447Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "print(torch.cuda.device_count())\n",
    "if torch.cuda.device_count() > 0:\n",
    "    for i in range(torch.cuda.device_count()):\n",
    "        print(torch.cuda.get_device_name(i))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-27T12:08:17.100545Z",
     "start_time": "2019-06-27T12:08:17.096959Z"
    }
   },
   "source": [
    "### Test Spacy GPU Support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:16:09.116943Z",
     "start_time": "2020-01-28T19:16:09.112567Z"
    }
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "spacy.require_gpu()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test XGBoost GPU Support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:16:57.833410Z",
     "start_time": "2020-01-28T19:16:57.020798Z"
    }
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.datasets import load_boston\n",
    "\n",
    "boston = load_boston()\n",
    "\n",
    "# XGBoost API example\n",
    "params = {'tree_method': 'gpu_hist', 'max_depth': 3, 'learning_rate': 0.1}\n",
    "dtrain = xgb.DMatrix(boston.data, boston.target)\n",
    "xgb.train(params, dtrain, evals=[(dtrain, \"train\")])\n",
    "\n",
    "# sklearn API example\n",
    "gbm = xgb.XGBRegressor(silent=False, n_estimators=10, tree_method='gpu_hist')\n",
    "gbm.fit(boston.data, boston.target, eval_set=[(boston.data, boston.target)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test MXNet GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:16:54.782796Z",
     "start_time": "2020-01-28T19:16:49.563650Z"
    }
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "a = mx.nd.ones((2, 3), mx.gpu())\n",
    "b = a * 2 + 1\n",
    "b.asnumpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test RAPIDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install repids tools\n",
    "! conda install -c rapidsai -c nvidia -c conda-forge -c defaults rapids=0.11 python=3.7 cudatoolkit=10.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### cudf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:17:01.776038Z",
     "start_time": "2020-01-28T19:17:01.578164Z"
    }
   },
   "outputs": [],
   "source": [
    "import cudf, io, requests\n",
    "from io import StringIO\n",
    "\n",
    "url=\"https://github.com/plotly/datasets/raw/master/tips.csv\"\n",
    "content = requests.get(url).content.decode('utf-8')\n",
    "\n",
    "tips_df = cudf.read_csv(StringIO(content))\n",
    "tips_df['tip_percentage'] = tips_df['tip']/tips_df['total_bill']*100\n",
    "\n",
    "# display average tip by dining party size\n",
    "print(tips_df.groupby('size').tip_percentage.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### cuml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-27T15:13:40.847239Z",
     "start_time": "2019-06-27T15:13:39.585522Z"
    }
   },
   "outputs": [],
   "source": [
    "import cudf\n",
    "from cuml.cluster import DBSCAN\n",
    "\n",
    "# Create and populate a GPU DataFrame\n",
    "gdf_float = cudf.DataFrame()\n",
    "gdf_float['0'] = [1.0, 2.0, 5.0]\n",
    "gdf_float['1'] = [4.0, 2.0, 1.0]\n",
    "gdf_float['2'] = [4.0, 2.0, 1.0]\n",
    "\n",
    "# Setup and fit clusters\n",
    "dbscan_float = DBSCAN(eps=1.0, min_samples=1)\n",
    "dbscan_float.fit(gdf_float)\n",
    "\n",
    "print(dbscan_float.labels_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### custrings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-27T15:13:44.527658Z",
     "start_time": "2019-06-27T15:13:44.297279Z"
    }
   },
   "outputs": [],
   "source": [
    "import nvstrings, nvcategory\n",
    "import requests\n",
    "\n",
    "url=\"https://github.com/plotly/datasets/raw/master/tips.csv\"\n",
    "content = requests.get(url).content.decode('utf-8')\n",
    "\n",
    "#split content into a list, remove header\n",
    "host_lines = content.strip().split('\\n')[1:]\n",
    "\n",
    "#copy strings to gpu\n",
    "gpu_lines = nvstrings.to_device(host_lines)\n",
    "\n",
    "#split into columns on gpu\n",
    "gpu_columns = gpu_lines.split(',')\n",
    "gpu_day_of_week = gpu_columns[4]\n",
    "\n",
    "#use gpu `replace` to re-encode tokens on GPU\n",
    "for idx, day in enumerate(['Sun', 'Mon', 'Tues', 'Wed', 'Thur', 'Fri', 'Sat']):\n",
    "    gpu_day_of_week = gpu_day_of_week.replace(day, str(idx))\n",
    "\n",
    "# or, use nvcategory's builtin GPU categorization\n",
    "cat = nvcategory.from_strings(gpu_columns[4])\n",
    "\n",
    "# copy category keys to host and print\n",
    "print(cat.keys())\n",
    "\n",
    "# copy \"cleaned\" strings to host and print\n",
    "print(gpu_day_of_week)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### dask-cuda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-27T15:13:50.063026Z",
     "start_time": "2019-06-27T15:13:46.633674Z"
    }
   },
   "outputs": [],
   "source": [
    "from dask_cuda import LocalCUDACluster\n",
    "from dask.distributed import Client\n",
    "\n",
    "cluster = LocalCUDACluster(local_dir=\"/tmp/dask\")\n",
    "client = Client(cluster)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test PyCuda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:17:06.287443Z",
     "start_time": "2020-01-28T19:17:06.139908Z"
    }
   },
   "outputs": [],
   "source": [
    "import pycuda.driver as cuda\n",
    "import pycuda.autoinit\n",
    "from pycuda.compiler import SourceModule\n",
    "import numpy\n",
    "a = numpy.random.randn(4,4)\n",
    "a = a.astype(numpy.float32)\n",
    "a_gpu = cuda.mem_alloc(a.nbytes)\n",
    "cuda.memcpy_htod(a_gpu, a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test pynvml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:17:10.720323Z",
     "start_time": "2020-01-28T19:17:10.699234Z"
    }
   },
   "outputs": [],
   "source": [
    "from pynvml import *\n",
    "nvmlInit()\n",
    "try:\n",
    "    deviceCount = nvmlDeviceGetCount()\n",
    "    for i in range(deviceCount):\n",
    "        handle = nvmlDeviceGetHandleByIndex(i)\n",
    "        print(\"Device\", i, \":\", nvmlDeviceGetName(handle))\n",
    "except NVMLError as error:\n",
    "    print(error)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test py3nvml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:17:15.229828Z",
     "start_time": "2020-01-28T19:17:13.835647Z"
    }
   },
   "outputs": [],
   "source": [
    "import py3nvml\n",
    "free_gpus = py3nvml.get_free_gpus()\n",
    "if True not in free_gpus:\n",
    "    print('No free gpus found')\n",
    "!py3smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-27T15:00:46.343746Z",
     "start_time": "2019-06-27T15:00:46.340809Z"
    }
   },
   "source": [
    "### Test gputil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:17:22.347471Z",
     "start_time": "2020-01-28T19:17:22.250308Z"
    }
   },
   "outputs": [],
   "source": [
    "import GPUtil\n",
    "GPUtil.showUtilization()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test GPUStat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:17:25.983522Z",
     "start_time": "2020-01-28T19:17:24.574825Z"
    }
   },
   "outputs": [],
   "source": [
    "!gpustat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Relase GPU Memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:26:29.083793Z",
     "start_time": "2020-01-28T19:26:21.749375Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "def consume_gpu(n):\n",
    "    return torch.ones((n,n)).cuda()\n",
    "!gpustat\n",
    "consume_gpu(1000)\n",
    "!gpustat\n",
    "torch.cuda.empty_cache()\n",
    "!gpustat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:26:40.476523Z",
     "start_time": "2020-01-28T19:26:37.016735Z"
    }
   },
   "outputs": [],
   "source": [
    "from numba import cuda\n",
    "cuda.select_device(0)\n",
    "cuda.close()\n",
    "cuda.select_device(0)\n",
    "!gpustat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Links & Resources\n",
    "- https://docs.fast.ai/dev/gpu.html\n",
    "- https://docs.fast.ai/troubleshoot.html#memory-leakage-on-exception"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optional Installs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Starspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:20:58.043717Z",
     "start_time": "2020-01-28T19:19:53.199407Z"
    }
   },
   "outputs": [],
   "source": [
    "# Install starspace\n",
    "!/resources/tools/starspace.sh --install\n",
    "# Starspace command line instructions:\n",
    "!starspace --help"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fasttext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:21:19.962934Z",
     "start_time": "2020-01-28T19:20:58.046231Z"
    }
   },
   "outputs": [],
   "source": [
    "# Install fasttext\n",
    "!/resources/tools/fasttext.sh --install\n",
    "# Starspace command line instructions:\n",
    "!fasttext --help"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pillow SIMD\n",
    "Replacement of pillow for faster image processing. More information here: https://docs.fast.ai/performance.html#faster-image-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:19:53.196988Z",
     "start_time": "2020-01-28T19:17:38.285991Z"
    }
   },
   "outputs": [],
   "source": [
    "# Install pillow simd\n",
    "!/resources/tools/pillow-simd.sh --install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:29:56.017190Z",
     "start_time": "2020-01-28T19:29:55.953990Z"
    }
   },
   "outputs": [],
   "source": [
    "# Check pillow install\n",
    "# You most likely need to restart the kernel after pillow simd installation\n",
    "\n",
    "# Temp workaround to fix missing PILLOW_VERSION on 7.0.0\n",
    "from PIL import Image\n",
    "Image.PILLOW_VERSION = PIL.__version__\n",
    "\n",
    "import fastai.utils\n",
    "fastai.utils.check_perf()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNTK\n",
    "\n",
    "_CNTK does not work with Python 3.7_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-26T11:58:04.999577Z",
     "start_time": "2020-01-26T11:58:02.217186Z"
    }
   },
   "outputs": [],
   "source": [
    "# Install cntk\n",
    "!/resources/tools/cntk.sh --install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-15T23:30:31.500021Z",
     "start_time": "2019-09-15T23:30:30.507163Z"
    }
   },
   "outputs": [],
   "source": [
    "import cntk\n",
    "print(cntk.__version__)\n",
    "cntk.minus([1, 2, 3], [4, 5, 6]).eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-15T23:30:37.384223Z",
     "start_time": "2019-09-15T23:30:33.656881Z"
    }
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import cntk as C\n",
    "from cntk.learners import sgd\n",
    "from cntk.logging import ProgressPrinter\n",
    "from cntk.layers import Dense, Sequential\n",
    "\n",
    "def generate_random_data(sample_size, feature_dim, num_classes):\n",
    "    # Create synthetic data using NumPy.\n",
    "    Y = np.random.randint(size=(sample_size, 1), low=0, high=num_classes)\n",
    "\n",
    "    # Make sure that the data is separable\n",
    "    X = (np.random.randn(sample_size, feature_dim) + 3) * (Y + 1)\n",
    "    X = X.astype(np.float32)\n",
    "    # converting class 0 into the vector \"1 0 0\",\n",
    "    # class 1 into vector \"0 1 0\", ...\n",
    "    class_ind = [Y == class_number for class_number in range(num_classes)]\n",
    "    Y = np.asarray(np.hstack(class_ind), dtype=np.float32)\n",
    "    return X, Y\n",
    "\n",
    "def ffnet():\n",
    "    inputs = 2\n",
    "    outputs = 2\n",
    "    layers = 2\n",
    "    hidden_dimension = 50\n",
    "\n",
    "    # input variables denoting the features and label data\n",
    "    features = C.input_variable((inputs), np.float32)\n",
    "    label = C.input_variable((outputs), np.float32)\n",
    "\n",
    "    # Instantiate the feedforward classification model\n",
    "    my_model = Sequential ([\n",
    "                    Dense(hidden_dimension, activation=C.sigmoid),\n",
    "                    Dense(outputs)])\n",
    "    z = my_model(features)\n",
    "\n",
    "    ce = C.cross_entropy_with_softmax(z, label)\n",
    "    pe = C.classification_error(z, label)\n",
    "\n",
    "    # Instantiate the trainer object to drive the model training\n",
    "    lr_per_minibatch = C.learning_parameter_schedule(0.125)\n",
    "    progress_printer = ProgressPrinter(0)\n",
    "    trainer = C.Trainer(z, (ce, pe), [sgd(z.parameters, lr=lr_per_minibatch)], [progress_printer])\n",
    "\n",
    "    # Get minibatches of training data and perform model training\n",
    "    minibatch_size = 25\n",
    "    num_minibatches_to_train = 1024\n",
    "\n",
    "    aggregate_loss = 0.0\n",
    "    for i in range(num_minibatches_to_train):\n",
    "        train_features, labels = generate_random_data(minibatch_size, inputs, outputs)\n",
    "        # Specify the mapping of input variables in the model to actual minibatch data to be trained with\n",
    "        trainer.train_minibatch({features : train_features, label : labels})\n",
    "        sample_count = trainer.previous_minibatch_sample_count\n",
    "        aggregate_loss += trainer.previous_minibatch_loss_average * sample_count\n",
    "\n",
    "    last_avg_error = aggregate_loss / trainer.total_number_of_samples_seen\n",
    "\n",
    "    test_features, test_labels = generate_random_data(minibatch_size, inputs, outputs)\n",
    "    avg_error = trainer.test_minibatch({features : test_features, label : test_labels})\n",
    "    print(' error rate on an unseen minibatch: {}'.format(avg_error))\n",
    "    return last_avg_error, avg_error\n",
    "\n",
    "np.random.seed(98052)\n",
    "ffnet()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test CNTK GPU Support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-15T23:31:07.572896Z",
     "start_time": "2019-09-15T23:31:07.399522Z"
    }
   },
   "outputs": [],
   "source": [
    "from cntk.device import try_set_default_device, gpu\n",
    "try_set_default_device(gpu(0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:25:11.394888Z",
     "start_time": "2020-01-28T19:24:57.763373Z"
    }
   },
   "outputs": [],
   "source": [
    "!pip install -U catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:25:14.760325Z",
     "start_time": "2020-01-28T19:25:14.592772Z"
    }
   },
   "outputs": [],
   "source": [
    "from catboost import CatBoostClassifier\n",
    "# Initialize data\n",
    "cat_features = [0,1,2]\n",
    "train_data = [[\"a\",\"b\",1,4,5,6],[\"a\",\"b\",4,5,6,7],[\"c\",\"d\",30,40,50,60]]\n",
    "train_labels = [1,1,-1]\n",
    "test_data = [[\"a\",\"b\",2,4,6,8],[\"a\",\"d\",1,4,50,60]]\n",
    "# Initialize CatBoostClassifier\n",
    "model = CatBoostClassifier(iterations=2, \n",
    "                           learning_rate=1, \n",
    "                           depth=2, \n",
    "                           loss_function='Logloss', \n",
    "                           train_dir=\"/tmp/catboost/\")\n",
    "# Fit model\n",
    "model.fit(train_data, train_labels, cat_features)\n",
    "# Get predicted classes\n",
    "preds_class = model.predict(test_data)\n",
    "# Get predicted probabilities for each class\n",
    "preds_proba = model.predict_proba(test_data)\n",
    "# Get predicted RawFormulaVal\n",
    "preds_raw = model.predict(test_data, prediction_type='RawFormulaVal')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test Catboost GPU Support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-28T19:25:27.958583Z",
     "start_time": "2020-01-28T19:25:17.441136Z"
    }
   },
   "outputs": [],
   "source": [
    "from catboost import CatBoostClassifier\n",
    "\n",
    "train_data = [[0, 3],\n",
    "              [4, 1],\n",
    "              [8, 1],\n",
    "              [9, 1]]\n",
    "train_labels = [0, 0, 1, 1]\n",
    "\n",
    "model = CatBoostClassifier(iterations=1000, \n",
    "                           task_type = \"GPU\",\n",
    "                           devices='0',\n",
    "                           train_dir=\"/tmp/catboost/\")\n",
    "model.fit(train_data,\n",
    "          train_labels,\n",
    "          verbose = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
